{
 "cells": [
  {
   "cell_type": "raw",
   "id": "027c4182-c168-4c53-a79c-670a7ace286f",
   "metadata": {},
   "source": [
    "%cd /project/train/v5ds/\n",
    "!python3 gen_2d.py\n",
    "!python3 gen_mask.py\n",
    "\n",
    "!python3 trainds.py --weights models/yolov5s.pt --device 0  --batch-size=1 --workers 4 \\\n",
    "--project /project/train/models/ --name o  --epoch 30 --data data/1441.yaml\n",
    "# --exist-ok\n",
    "# --weights runs/best.pt --data data/car.yaml --workers 1 --batch-size 16 --project /project/train/models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1be16fc7-95ff-445b-8c51-a053352c16ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/project/train/v5ds\n",
      "xmls: 41\n",
      "xmls: 5\n",
      "func is running 0:00:00.058130 s\n",
      "rm: cannot remove '/home/data/yolo3d': No such file or directory\n",
      "^C\n",
      "Traceback (most recent call last):\n",
      "  File \"gen_mask.py\", line 215, in <module>\n",
      "    cv2.imwrite(f\"{ROOT_PATH}/yolo3d/labels/{img_cnt}.png\",mask)\n",
      "KeyboardInterrupt\n",
      "\u001b[34m\u001b[1mtrainds: \u001b[0mweights=/project/train/models/best.pt, cfg=models/yolov5s.yaml, segcfg=models/segheads.yaml, data=data/1441.yaml, hyp=data/hyps/hyp.scratch-high.yaml, epochs=100, batch_size=1, imgsz=512, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, evolve=None, bucket=, cache=None, image_weights=False, device=0, multi_scale=False, single_cls=False, adam=False, sync_bn=False, workers=1, project=/project/train/runs/, name=o, exist_ok=True, quad=False, linear_lr=False, label_smoothing=0.0, patience=100, freeze=0, save_period=-1, local_rank=-1, entity=None, upload_dataset=False, bbox_interval=-1, artifact_alias=latest\n",
      "YOLOv5 🚀 ec0f7e0 torch 1.10.0+cu102 CUDA:0 (GeForce RTX 2080 Ti, 11019MiB)\n",
      "\n",
      "\u001b[34m\u001b[1mhyperparameters: \u001b[0mlr0=0.01, lrf=0.2, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.3, cls_pw=1.0, obj=0.7, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.9, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.1, copy_paste=0.1\n",
      "\u001b[34m\u001b[1mWeights & Biases: \u001b[0mrun 'pip install wandb' to automatically track and visualize YOLOv5 🚀 runs (RECOMMENDED)\n",
      "\u001b[34m\u001b[1mTensorBoard: \u001b[0mStart with 'tensorboard --logdir /project/train/runs', view at http://localhost:6006/\n",
      "using pretained weights...\n"
     ]
    }
   ],
   "source": [
    "%cd /project/train/v5ds/\n",
    "!python3 gen_2d.py\n",
    "!python3 gen_mask.py\n",
    "\n",
    "!python3 trainds.py --weights /project/train/models/best.pt --device 0  --batch-size=1 --workers 1 \\\n",
    "--project /project/train/runs/ --name o  --epoch 100 --data data/1441.yaml  --exist-ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d002913b-15c9-4945-94cf-5bfc08320a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git reset --hard origin/master\n",
    "!git pull\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "775d4d6c-7665-4b8b-a856-c747846ceeff",
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /project/train/src_repo/\n",
    "!cp -r ./* /project/ev_sdk/src/\n",
    "!rm /project/ev_sdk/src/ji.py\n",
    "!ln ./ji.py /project/ev_sdk/src/ji.py\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b6fe8b31-e48b-4e3e-a7f3-4031e225507d",
   "metadata": {},
   "source": [
    "# 测试 训练\n",
    "!cd && sh /project/train/src_repo/run.sh\n",
    "# sh project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "58b11929-2224-4cff-8843-018f29226b4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/project/train/v5ds\n",
      "ok\n"
     ]
    }
   ],
   "source": [
    "#拷贝到训练目录\n",
    "%cd /project/train/v5ds\n",
    "!rm -r /project/train/src_repo/*\n",
    "!mkdir -p  /project/train/src_repo/\n",
    "!cp -r ./* /project/train/src_repo/\n",
    "!echo ok"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b9332e87-4692-4d9a-af20-62380bced48a",
   "metadata": {},
   "source": [
    "%cd /project/train/src_repo/ds5/\n",
    "!python3 dete_color.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdf8205b-d595-4986-863f-6d0f6a171a5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sh /project/train/src_repo/run.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e834aae9-ad4a-4f55-b98e-67d3dc026eec",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -r /project/train/models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5154a95c-8e5c-41cc-b934-41fa178620c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import psutil\n",
    "import os,datetime,time\n",
    "\n",
    "def getMemCpu():\n",
    " data = psutil.virtual_memory()\n",
    " total = data.total #总内存,单位为byte\n",
    " free = data.available #可以内存\n",
    " memory = \"Memory total %dG/%dG usage:%d \"%(total>>30,free>>30,int(round(data.percent)))+\"%\"+\" \"\n",
    " cpu = \"CPU:%0.2f\"%psutil.cpu_percent(interval=1)+\"%\"\n",
    " return memory+cpu\n",
    "ret=getMemCpu()\n",
    "print(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b7c963e5-14f9-402f-89fc-ceb82e79f3eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install GPUtil\n",
    "!cp /home/data/1441/* /home/data/1442"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49b86729-e6fb-47b7-84d5-040ee34c5115",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "print(time.strftime('%H:%M:%S', time.localtime(time.time())))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a38144b0-b07b-46e5-aa9e-7a32f14c6fb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from pynvml import *\n",
    "\n",
    "\n",
    "def get_nv_smi():\n",
    "    handle = nvmlDeviceGetHandleByIndex(0)  # 0 是 GPU id\n",
    "    meminfo = nvmlDeviceGetMemoryInfo(handle)\n",
    "    print(meminfo.total / 1024 / 1024)  # 显存大小\n",
    "    print(meminfo.used / 1024 / 1024)  # 单位：字节bytes，所以要想得到以兆M为单位就需要除以1024**2\n",
    "    print(meminfo.free / 1024 / 1024)  # 显卡剩余显存大小\n",
    "    print(nvmlDeviceGetCount())  # 查询显示GPU数目\n",
    "\n",
    "\n",
    "def nvidia_info():\n",
    "    nvidia_dict = {\n",
    "        \"state\": True,\n",
    "        \"nvidia_version\": \"\",\n",
    "        \"nvidia_count\": 0,\n",
    "        \"gpus\": []\n",
    "    }\n",
    "    try:\n",
    "        nvmlInit()\n",
    "        nvidia_dict[\"nvidia_version\"] = nvmlSystemGetDriverVersion()\n",
    "        nvidia_dict[\"nvidia_count\"] = nvmlDeviceGetCount()\n",
    "        for i in range(nvidia_dict[\"nvidia_count\"]):\n",
    "            handle = nvmlDeviceGetHandleByIndex(i)\n",
    "            memory_info = nvmlDeviceGetMemoryInfo(handle)\n",
    "            gpu = {\n",
    "                \"gpu_name\": nvmlDeviceGetName(handle),\n",
    "                \"total\": memory_info.total,\n",
    "                \"free\": memory_info.free,\n",
    "                \"used\": memory_info.used,\n",
    "                \"temperature\": f\"{nvmlDeviceGetTemperature(handle, 0)}℃\",\n",
    "                \"powerStatus\": nvmlDeviceGetPowerState(handle)\n",
    "            }\n",
    "            nvidia_dict['gpus'].append(gpu)\n",
    "    except NVMLError as _:\n",
    "        nvidia_dict[\"state\"] = False\n",
    "    except Exception as _:\n",
    "        nvidia_dict[\"state\"] = False\n",
    "    finally:\n",
    "        try:\n",
    "            nvmlShutdown()\n",
    "        except:\n",
    "            pass\n",
    "    return nvidia_dict\n",
    "import time\n",
    "\n",
    "def check_gpu_mem_usedRate():\n",
    "    max_rate = 0.0\n",
    "\n",
    "    for i in range(100):\n",
    "        info = nvidia_info()\n",
    "        print(info)\n",
    "        used = info['gpus'][0]['used']\n",
    "        total = info['gpus'][0]['total']\n",
    "        temperature = info['gpus'][0]['temperature']\n",
    "        print(f\"GPU0 used: {used}, total: {total}, 使用率：{used/total}\")\n",
    "        if used / total > max_rate:\n",
    "            max_rate = used / total\n",
    "        print(\"GPU0 最大使用率：\", max_rate)\n",
    "        time.sleep(0.2)\n",
    "\n",
    "\n",
    "        # break\n",
    "        # if __name__ == '__main__':\n",
    "nvmlInit()\n",
    "get_nv_smi()\n",
    "nvidia_info()\n",
    "check_gpu_mem_usedRate()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28b1b40e-c703-4c6d-96d5-785faa2ead09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install pynvml\n",
    "\n",
    "from multiprocessing import shared_memory\n",
    "\n",
    "\n",
    "shm = shared_memory.SharedMemory(name='shmTest555666', create=True, size=10)\n",
    "\n",
    "buf = shm.buf\n",
    "print('Process A, buf len: {}, {}'.format(len(buf), shm.size))\n",
    "print('shm name: {}'.format(shm.name))\n",
    "\n",
    "buf[:4] = bytearray([22, 33, 44, 55])\n",
    "buf[4] = 100\n",
    "\n",
    "\n",
    "while True:\n",
    "    if buf[4] == 200: # 当buf[4]变成200，就结束循环\n",
    "        break\n",
    "\n",
    "\n",
    "shm.close() # 关闭共享内存\n",
    "shm.unlink() # 释放共享内存，也可以由B进程释放\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebc903ad-e9e6-4fc5-a984-0e489fa53709",
   "metadata": {},
   "outputs": [],
   "source": [
    "import GPUtil\n",
    "import time\n",
    "time_x = []\n",
    "gpu_y = []\n",
    "\n",
    "def gpu_util_timer(self):\n",
    "    for n in range(10):\n",
    "        Graph_Util.gpu_y.append(GPUtil.showUtilization())\n",
    "        Graph_Util.time_x.append(n)\n",
    "        time.sleep(1)\n",
    "    print('gpu done')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85af138f-292c-49d6-aed3-ddcd637bcbfa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
